{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import pandas as pd\n",
    "\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset glue (/Users/raviraja/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
     ]
    }
   ],
   "source": [
    "cola_dataset = load_dataset('glue', 'cola')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence', 'label', 'idx'],\n",
       "        num_rows: 8551\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence', 'label', 'idx'],\n",
       "        num_rows: 1043\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence', 'label', 'idx'],\n",
       "        num_rows: 1063\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cola_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = cola_dataset['train']\n",
    "val_dataset = cola_dataset['validation']\n",
    "test_dataset = cola_dataset['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8551, 1043, 1063)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataset), len(val_dataset), len(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'idx': 0,\n",
       " 'label': 1,\n",
       " 'sentence': \"Our friends won't buy this analysis, let alone the next one we propose.\"}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'idx': 0,\n",
       " 'label': 1,\n",
       " 'sentence': 'The sailors rode the breeze clear of the rocks.'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'idx': 0, 'label': -1, 'sentence': 'Bill whistled past the house.'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence': Value(dtype='string', id=None),\n",
       " 'label': ClassLabel(num_classes=2, names=['unacceptable', 'acceptable'], names_file=None, id=None),\n",
       " 'idx': Value(dtype='int32', id=None)}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c681f26df104422a4c21a216b351949",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'idx': [0, 1, 2, 3, 4],\n",
       " 'label': [1, 1, 1, 1, 1],\n",
       " 'sentence': [\"Our friends won't buy this analysis, let alone the next one we propose.\",\n",
       "  \"One more pseudo generalization and I'm giving up.\",\n",
       "  \"One more pseudo generalization or I'm giving up.\",\n",
       "  'The more we study verbs, the crazier they get.',\n",
       "  'Day by day the facts are getting murkier.']}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.filter(lambda example: example['label'] == train_dataset.features['label'].str2int('acceptable'))[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7276a21736814e29b7df2af0bdee2dab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'idx': [18, 20, 22, 23, 25],\n",
       " 'label': [0, 0, 0, 0, 0],\n",
       " 'sentence': ['They drank the pub.',\n",
       "  'The professor talked us.',\n",
       "  'We yelled ourselves.',\n",
       "  'We yelled Harry hoarse.',\n",
       "  'Harry coughed himself.']}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.filter(lambda example: example['label'] == train_dataset.features['label'].str2int('unacceptable'))[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"google/bert_uncased_L-2_H-128_A-2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = cola_dataset['train']\n",
    "val_dataset = cola_dataset['validation']\n",
    "test_dataset = cola_dataset['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizerFast(name_or_path='google/bert_uncased_L-2_H-128_A-2', vocab_size=30522, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our friends won't buy this analysis, let alone the next one we propose.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2256, 2814, 2180, 1005, 1056, 4965, 2023, 4106, 1010, 2292, 2894, 1996, 2279, 2028, 2057, 16599, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_dataset[0]['sentence'])\n",
    "tokenizer(train_dataset[0]['sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"[CLS] our friends won't buy this analysis, let alone the next one we propose. [SEP]\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenizer(train_dataset[0]['sentence'])['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(examples):\n",
    "    return tokenizer(\n",
    "            examples[\"sentence\"],\n",
    "            truncation=True,\n",
    "            padding=\"max_length\",\n",
    "            max_length=512,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5205de7df394d5a800f2ee94d3c9106",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_dataset = train_dataset.map(encode, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         ...,\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0]]),\n",
       " 'input_ids': tensor([[  101,  2256,  2814,  ...,     0,     0,     0],\n",
       "         [  101,  2028,  2062,  ...,     0,     0,     0],\n",
       "         [  101,  2028,  2062,  ...,     0,     0,     0],\n",
       "         ...,\n",
       "         [  101,  5965, 12808,  ...,     0,     0,     0],\n",
       "         [  101,  2198, 10948,  ...,     0,     0,     0],\n",
       "         [  101,  3021, 24471,  ...,     0,     0,     0]]),\n",
       " 'label': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0,\n",
       "         1, 0, 0, 1, 1, 1, 1, 1])}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(dataloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([32, 512]) torch.Size([32, 512]) torch.Size([32])\n",
      "torch.Size([7, 512]) torch.Size([7, 512]) torch.Size([7])\n"
     ]
    }
   ],
   "source": [
    "for batch in dataloader:\n",
    "    print(batch['input_ids'].shape, batch['attention_mask'].shape, batch['label'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "day1",
   "language": "python",
   "name": "day1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
